% this file uses puddleworld_data_* files to generate results for the
% following experiment: 
% if we fix the number of interactions between us and the environment, how
% should we collect data (training/testing), so that evaluator algorithms
% can better predict Q-pi. Note that our data is collected by running pi
% and pi_e interchangably. We then decide how many points from each pi
% trajectory to include in the training set (look at variable into_traj). 
% we test the results for different interaction lengths (variable
% min_history_length). 
% we test these for 3 different evaluator algs: cmac, rbf and perfect. 
% 


%these are the steps we're doing here: 
%- load histories variable from file 
%- flatten histories into hflat 
%- create fixed testing using hflat
%- create different training data
%- call vary_algorithm_gold on different training testing 




%load histories 
load('puddleworld_data_perfect', 'histories_100'); 
histories = histories_100; 
%make flat history 
hflat = flattenHistory(histories); 

TRIALS = 10;        %how many times do we do the whole experiment?
for x=1:TRIALS
    display(['trial number ' num2str(x)]); 
    [t testing] = getTrainingTesting(hflat, 1, 200, -1); 
    t = floor(rand(200,1)*4+1); 
    testing = [testing t];          %add actions to the testing set
    clear t


    all_train = {}; 

    min_history_length = [1 5 10 20]; %how many of mini histories to include in the training
    into_traj = [1 10 20 -1];   %how to generate training set

    for i=1:length(min_history_length)
        ht = flattenHistory(histories, min_history_length(i)); 
        for j=1:length(into_traj)
            all_train{i}{j} = getTrainingTesting(ht, -1, 2, into_traj(j));        
        end
    end

    clear i j ht


    display('Finished constructing training/testing. Now Evaluating'); 


%    ms = {} 
    global qgold
    qgold = [];         %inside vary_algorithm_gold, we only compute qgold when it's empty. We want it to compute the first time. 
    for i=1:4
        for j=1:4 
           ms{x}{i}{j} =  vary_algorithm_gold('puddleworld', final, all_train{i}{j}, testing)
        end

    end
end % each trial 

cmac = zeros(4,4); 
rbf = zeros(4,4); 
perfect = zeros(4,4); 

for i=1:4
    for j=1:4
        for x=1:TRIALS 
            cmac(i,j) = cmac(i,j) +  ms{x}{i}{j}(1,1)/TRIALS;
            rbf(i,j) = rbf(i,j) + ms{x}{i}{j}(2,1)/TRIALS; 
            perfect(i,j) = perfect(i,j) + ms{x}{i}{j}(3,1)/TRIALS; 
        end
    end
end

clear TRIALS
clear i j x 







